importfeed: Support --json and --json-error-messages and --json-progress

Sponsored-By: the NIH-funded NICEMAN (ReproNim TR&D3) project

diff --git a/CHANGELOG b/CHANGELOG
index 3ff51db49d51cb0474638e1aa3d80ab81e2e0376..fb19b60898836d9afe414e0d10a940dfd2abcd91 100644 (file)
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -38,15 +38,16 @@ git-annex (10.20230408) UNRELEASED; urgency=medium
   * initremote: Avoid creating a remote that is not encrypted when gpg is
     broken.
   * Support --json and --json-error-messages in more commands
-    (addunused, dead, describe, dropunused, expire, fix, init, log, migrate,
-    reinit, reinject, rekey, renameremote, rmurl, semitrust, setpresentkey,
-    trust, unannex, undo, untrust, unused)
+    (addunused, dead, describe, dropunused, expire, fix, importfeed, init,
+    log, migrate, reinit, reinject, rekey, renameremote, rmurl, semitrust,
+    setpresentkey, trust, unannex, undo, untrust, unused)
   * log: When --raw-date is used, display only seconds from the epoch, as
     documented, omitting a trailing "s" that was included in the output
     before.
   * addunused: Displays the names of the files that it adds.
   * reinject: Fix support for operating on multiple pairs of files and keys.
   * importfeed: Support -J
+  * importfeed: Support --json-progress
 
  -- Joey Hess <id@joeyh.name>  Sat, 08 Apr 2023 13:57:18 -0400
 

diff --git a/Command/ImportFeed.hs b/Command/ImportFeed.hs
index 55a85647030ccb0b99f5ac421efa0a43c2bc6131..2901e7628f4a67a618806a99fa9db7c1c8a87f6d 100644 (file)
--- a/Command/ImportFeed.hs
+++ b/Command/ImportFeed.hs
@@ -54,9 +54,11 @@ import Logs
 import qualified Utility.RawFilePath as R
 
 cmd :: Command
-cmd = notBareRepo $ withAnnexOptions [jobsOption, backendOption] $
+cmd = notBareRepo $ withAnnexOptions os $
        command "importfeed" SectionCommon "import files from podcast feeds"
                (paramRepeating paramUrl) (seek <$$> optParser)
+  where
+       os = [jobsOption, jsonOptions, jsonProgressOption, backendOption]
 
 data ImportFeedOptions = ImportFeedOptions
        { feedUrls :: CmdParams
@@ -139,7 +141,7 @@ getFeed
        -> TMVar (M.Map URLString (Maybe (Maybe [ToDownload])))
        -> CommandStart
 getFeed url st =
-       starting "importfeed" (ActionItemOther (Just (UnquotedString url))) (SeekInput []) $
+       starting "importfeed" (ActionItemOther (Just (UnquotedString url))) (SeekInput [url]) $
                get `onException` recordfail
   where
        record v = liftIO $ atomically $ do
@@ -210,9 +212,12 @@ getCache :: Maybe String -> Annex Cache
 getCache opttemplate = ifM (Annex.getRead Annex.force)
        ( ret S.empty S.empty
        , do
-               showStartMessage (StartMessage "importfeed" (ActionItemOther (Just "gathering known urls")) (SeekInput []))
+               j <- jsonOutputEnabled
+               unless j $
+                       showStartMessage (StartMessage "importfeed" (ActionItemOther (Just "gathering known urls")) (SeekInput []))
                (us, is) <- knownItems
-               showEndOk
+               unless j
+                       showEndOk
                ret (S.fromList us) (S.fromList is)
        )
   where
@@ -295,7 +300,7 @@ startDownload addunlockedmatcher opts cache cv todownload = case location todown
        recordsuccess = liftIO $ atomically $ putTMVar cv True
        
        startdownloadenclosure :: URLString -> CommandStart
-       startdownloadenclosure url = checkknown url $ startUrlDownload cv url $
+       startdownloadenclosure url = checkknown url $ startUrlDownload cv todownload url $
                downloadEnclosure addunlockedmatcher opts cache cv todownload url 
 
        knownitemid = case getItemId (item todownload) of
@@ -306,7 +311,7 @@ startDownload addunlockedmatcher opts cache cv todownload = case location todown
        downloadmedia linkurl mediaurl mediakey
                | rawOption (downloadOptions opts) = startdownloadlink
                | otherwise = ifM (youtubeDlSupported linkurl)
-                       ( startUrlDownload cv linkurl $
+                       ( startUrlDownload cv todownload linkurl $
                                withTmpWorkDir mediakey $ \workdir -> do
                                        dl <- youtubeDl linkurl (fromRawFilePath workdir) nullMeterUpdate
                                        case dl of
@@ -336,7 +341,7 @@ startDownload addunlockedmatcher opts cache cv todownload = case location todown
        addmediafast linkurl mediaurl mediakey =
                ifM (pure (not (rawOption (downloadOptions opts)))
                     <&&> youtubeDlSupported linkurl)
-                       ( startUrlDownload cv linkurl $ do
+                       ( startUrlDownload cv todownload linkurl $ do
                                runDownload todownload linkurl ".m" cache cv $ \f ->
                                        checkCanAdd (downloadOptions opts) f $ \canadd -> do
                                                addWorkTree canadd addunlockedmatcher webUUID mediaurl f mediakey Nothing
@@ -453,10 +458,10 @@ runDownload todownload url extension cache cv getter = do
                        , tryanother
                        )
 
-startUrlDownload :: TMVar Bool -> URLString -> CommandPerform -> CommandStart
-startUrlDownload cv url a = starting "addurl"
+startUrlDownload :: TMVar Bool -> ToDownload -> URLString -> CommandPerform -> CommandStart
+startUrlDownload cv todownload url a = starting "addurl"
        (ActionItemOther (Just (UnquotedString url)))
-       (SeekInput [])
+       (SeekInput [feedurl todownload])
        (a `onException` recordfailure)
   where
        recordfailure = do

diff --git a/doc/git-annex-importfeed.mdwn b/doc/git-annex-importfeed.mdwn
index afcc29a2205219f089db9d956d42536b9e10be4d..aa8e45b62dda97d8949499f3bdc807c16e4ad53e 100644 (file)
--- a/doc/git-annex-importfeed.mdwn
+++ b/doc/git-annex-importfeed.mdwn
@@ -112,6 +112,20 @@ resulting in the new url being downloaded to such a filename.
 
   Specifies which key-value backend to use.
 
+* `--json`
+
+  Enable JSON output. This is intended to be parsed by programs that use
+  git-annex. Each line of output is a JSON object.
+
+* `--json-progress`
+
+  Include progress objects in JSON output.
+
+* `--json-error-messages`
+
+  Messages that would normally be output to standard error are included in
+  the JSON instead.
+
 * Also the [[git-annex-common-options]](1) can be used.
 
 # SEE ALSO

diff --git a/doc/todo/--json_for_unannex__and_ideally_any_other_command_.mdwn b/doc/todo/--json_for_unannex__and_ideally_any_other_command_.mdwn
index 5ae0d171e3c1c41f163ee20c51c8ddb0c56c9f09..b165cb6ef7aea7e52da25513f22e35e6553f0a7d 100644 (file)
--- a/doc/todo/--json_for_unannex__and_ideally_any_other_command_.mdwn
+++ b/doc/todo/--json_for_unannex__and_ideally_any_other_command_.mdwn
@@ -36,6 +36,7 @@ These commands have been updated to support --json:
 * git-annex-reinit
 * git-annex-reinject
 * git-annex-renameremote
+* git-annex-importfeed
 
 Provisional list of commands that don't support --json and maybe should:
 
@@ -55,10 +56,6 @@ These commands could support json, but I punted:
 * git-annex-version (--raw already exists, and the output is fairly machine
   parseable already. It would be possible to jsonize the output to make it
   possibly more machine parseable. But I'm doubtful that would be useful.
-* git-annex-importfeed (implemented w/o using usual command actions,
-  which makes warning messages not get put in any particular json record.
-  Same problem would also need to be fixed for [[doc/todo/importfeed_parallell]]
-  btw.)
 
 These commands have been reviewed and should not support json: